Advanced Lane Finding Project

The goals / steps of this project are the following:

• Compute the camera calibration matrix and distortion coefficients given a set of chessboard images.
• Apply a distortion correction to raw images.
• Use color transforms, gradients, etc., to create a thresholded binary image.
• Apply a perspective transform to rectify binary image ("birds-eye view").
• Detect lane pixels and fit to find the lane boundary.
• Determine the curvature of the lane and vehicle position with respect to center.
• Warp the detected lane boundaries back onto the original image.
• Output visual display of the lane boundaries and numerical estimation of lane curvature and vehicle position.

---

Preperation and utilities

import numpy as np
import cv2
import glob
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
%matplotlib inline
#matplotlib qt

def show_images(images):

    num_img = len(images)
    #print(num_img)
    #plt.figure(figsize=(15,5))
    #fig, axes = plt.subplots(num_img, 1)
    #for axe, i in zip(axes.flat, range(num_img)):
    #    print(i)
    #    axe.imshow(images[i])
    #    #plt.imshow(images[i])
    for i in range(num_img):
        #print("i:",i)
        plt.figure(i)
        #plt.subplot(num_img, 1, i+1)
        
        plt.imshow(images[i])
        #plt.imshow(images[i])

#plt.show()

def show_two_images(img1, img2, title1="Original Image", title2="Result Image"):

    f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
    f.tight_layout()
    ax1.imshow(img1)
    ax1.set_title(title1, fontsize=50)
    ax2.imshow(img2)
    ax2.set_title(title2, fontsize=50)
    plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

def show_pair_images(images1, images2):
    for i in range(len(images1)):
        show_two_images(images1[i], imges2[i])

Camera calibration

First, I'll compute the camera calibration using chessboard images

# a function that takes an image, object points, and image points
# performs the camera calibration, image distortion correction and 
# returns the undistorted image
def cal_undistort(img, objpoints, imgpoints):
    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
    ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, gray.shape[::-1], None, None)
    img_dst = cv2.undistort(img, mtx, dist, None, mtx)
    return img_dst

def compute_camera_calibration(images,chessboard_size_x=9, chessboard_size_y=6):
    # prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
    objp = np.zeros((chessboard_size_y*chessboard_size_x,3), np.float32)
    objp[:,:2] = np.mgrid[0:chessboard_size_x,0:chessboard_size_y].T.reshape(-1,2)

    # Arrays to store object points and image points from all the images.
    objpoints = [] # 3d points in real world space
    imgpoints = [] # 2d points in image plane.

    # Make a list of calibration images
    images_converted = []

    # Step through the list and search for chessboard corners
    for fname in images:
        #print(fname)
        img = cv2.imread(fname)
        img_converted = np.copy(img)
        gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)

        # Find the chessboard corners
        ret, corners = cv2.findChessboardCorners(gray, (chessboard_size_x,chessboard_size_y),None)

        # If found, add object points, image points
        if ret == True:
            objpoints.append(objp)
            imgpoints.append(corners)

            # Draw and display the corners
            img_converted = cv2.drawChessboardCorners(img_converted, (chessboard_size_x,chessboard_size_y), corners, ret)
            images_converted.append(img_converted)
            
            # Undistort the original image using objpoints and imgpoints
            img_undistorted = cal_undistort(img, objpoints, imgpoints)
            show_two_images(img_converted, img_undistorted)

    return objpoints, imgpoints

camera_images = glob.glob('./camera_cal/calibration*.jpg')
objpoints, imgpoints = compute_camera_calibration(camera_images)

Correcting for Distortion

use chessboard images to obtain image points and object points, and then compute the calibration and undistortion.

test_img = cv2.imread('./camera_cal/calibration2.jpg')

undistorted = cal_undistort(test_img, objpoints, imgpoints)
show_two_images(test_img, undistorted, title2="Undistorted Image")

Magnitude of the Gradient

Color and Gradient

import numpy as np
import cv2
import matplotlib.pyplot as plt
import matplotlib.image as mpimg


# Edit this function to create your own pipeline.
def pipeline_thresh(img, s_thresh=(170, 255), sx_thresh=(20, 100)):
    img = np.copy(img)
    # Convert to HLS color space and separate the V channel
    hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS).astype(np.float)
    l_channel = hls[:,:,1]
    s_channel = hls[:,:,2]
    
    # Grayscale image
    # NOTE: we already saw that standard grayscaling lost color information for the lane lines
    # Explore gradients in other colors spaces / color channels to see what might work better
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)

    # Sobel x
    #sobelx = cv2.Sobel(l_channel, cv2.CV_64F, 1, 0) # Take the derivative in x
    sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0) # Take the derivative in x
    
    abs_sobelx = np.absolute(sobelx) # Absolute x derivative to accentuate lines away from horizontal
    scaled_sobel = np.uint8(255*abs_sobelx/np.max(abs_sobelx))
    
    # Threshold x gradient
    sxbinary = np.zeros_like(scaled_sobel)
    sxbinary[(scaled_sobel >= sx_thresh[0]) & (scaled_sobel <= sx_thresh[1])] = 1
    
    # Threshold color channel
    s_binary = np.zeros_like(s_channel)
    s_binary[(s_channel >= s_thresh[0]) & (s_channel <= s_thresh[1])] = 1
    # Stack each channel
    # Note color_binary[:, :, 0] is all 0s, effectively an all black image. It might
    # be beneficial to replace this channel with something else.
    color_binary = np.dstack(( np.zeros_like(sxbinary), sxbinary, s_binary)) * 255
    
    # Combine the two binary thresholds
    combined_binary = np.zeros_like(sxbinary)
    combined_binary[(s_binary == 1) | (sxbinary == 1)] = 1
    #combined_binary[(s_binary == 1)] = 1

    return combined_binary
    #return color_binary
    
    #return gray
    

#image = mpimg.imread('./test_images/test5.jpg')
test_image = mpimg.imread('./test_images/straight_lines1.jpg')

image_threshed = pipeline_thresh(test_image)

show_two_images(test_image, image_threshed, title2="Thresh Pipeline Result")

test_img = mpimg.imread('./issue_images/org_video_image_20171217_10_22_06.jpg')
test_output = pipeline_thresh(test_img, s_thresh=(170, 255), sx_thresh=(5, 100))
show_two_images(test_img, test_output, title2="Magnitude Image")

undistorted = cal_undistort(test_image, objpoints, imgpoints)
show_two_images(test_image, undistorted, title2="Undistorted Image")

Warp

Y_TOP = 435#440
Y_BTM = 668

X_CTR = 640#640 = 1280/2

X_TOP_WDT = 48#45#50#55#65#50#50#60#60
X_BTM_WDT = 1000#1280#1000#1000#900#850

# less space is used
Y_TOP = 450#435#440
X_TOP_WDT = 120#48#45#50#55#65#50#50#60#60


TOP_LEFT = (X_CTR - X_TOP_WDT/2 , Y_TOP)
TOP_RIGHT = (X_CTR + X_TOP_WDT/2, Y_TOP)
BTM_LEFT = (X_CTR - X_BTM_WDT/2 , Y_BTM)
BTM_RIGHT = (X_CTR + X_BTM_WDT/2, Y_BTM)

SRC_TRAPEZOID_ORG_ORDR = [TOP_LEFT, TOP_RIGHT, BTM_RIGHT, BTM_LEFT]
#SRC_TRAPEZOID_WRP = [TOP_LEFT, TOP_RIGHT, BTM_LEFT, BTM_RIGHT]

def pipeline_warp(image, offsetX = 50, offsetY = 0):
    offset = 100 # offset for dst points
    # Grab the image shape
    img_size = (image.shape[1], image.shape[0])
    w = img_size[1]
    h = img_size[1]

    offsetX = 30 # test
    
    src = np.float32(SRC_TRAPEZOID_ORG_ORDR)#SRC_RECTANGLE)
    #error: ..\..\..\modules\imgproc\src\imgwarp.cpp:6101: error: (-215) _src.total() > 0 in function cv::warpPerspective
    #src = np.float32([corners[0],corners[7],corners[40],corners[47]])#([[,],[,],[,],[,]])
    # c) define 4 destination points dst = np.float32([[,],[,],[,],[,]])
    # For destination points, I'm arbitrarily choosing some points to be
    # a nice fit for displaying our warped result 
    # again, not exact, but close enough for our purposes
    #dst = np.float32([[offset, offset], [img_size[0]-offset, offset], 
    #                     [img_size[0]-offset, img_size[1]-offset], 
    #                     [offset, img_size[1]-offset]])
    
    #dst = np.float32([[offsetX, offsetY], [w - offsetX, offsetY], 
    #                 [offsetX, h - offsetY], [w - offsetX, h - offsetY]])

    dst = np.float32([[offsetX, offsetY], [w - offsetX, offsetY], 
                      [w - offsetX, h - offsetY],
                      [offsetX, h - offsetY]])
        
    #dst = np.float32([[,],[,],[,],[,]])
    # d) use cv2.getPerspectiveTransform() to get M, the transform matrix
    M = cv2.getPerspectiveTransform(src, dst)
    # e) use cv2.warpPerspective() to warp your image to a top-down view
    #warped = cv2.warpPerspective(img, M, img_size, flags=cv2.INTER_LINEAR)
    
    warped = cv2.warpPerspective(image,M,(h, w))
    
    Minv = cv2.getPerspectiveTransform(dst, src)

    return warped, M, Minv

#test_img = mpimg.imread('./test_images/straight_lines1.jpg')
test_img = mpimg.imread('./test_images/test1.jpg')
image_threshed = pipeline_thresh(test_img)
image_warped, M, Minv = pipeline_warp(image_threshed)
show_two_images(test_img, image_threshed, title2="Binary Image")
show_two_images(test_img, image_warped, title2="Warped Image")

Finding the Lines

Tracking

TODO; Line class is not used.

# Define a class to receive the characteristics of each line detection
class Line():
    def __init__(self):
        # was the line detected in the last iteration?
        self.detected = False  
        # x values of the last n fits of the line
        self.recent_xfitted = [] 
        
        #average x values of the fitted line over the last n iterations
        self.bestx = None     
        #polynomial coefficients averaged over the last n iterations
        self.best_fit = None 
        
        #polynomial coefficients for the most recent fit
        self.current_fit = [np.array([False])]  
        self.prev_fit = [np.array([False])]  
        
        #radius of curvature of the line in some units
        self.radius_of_curvature = None 
        #distance in meters of vehicle center from the line
        self.line_base_pos = None 
        
        #difference in fit coefficients between last and new fits
        self.diffs = np.array([0,0,0], dtype='float') 
        
        #x values for detected line pixels
        self.allx = None  
        #y values for detected line pixels
        self.ally = None

    def calc_diffs():
        self.diffs[0] = self.current_fit[0] - self.prev_fit[0]
        self.diffs[1] = self.current_fit[1] - self.prev_fit[1]
        self.diffs[2] = self.current_fit[2] - self.prev_fit[2]
        
    def update_average_and_best(self):
        #average x values of the fitted line over the last n iterations
        self.bestx = None
        #polynomial coefficients averaged over the last n iterations
        self.best_fit = None 

    def compare_last_and_recent(self):
        #difference in fit coefficients between last and new fits
        self.diffs = np.array([0,0,0], dtype='float') 

    def add_recent_data(self, current_fit, detected_x, detected_y):
    
        # was the line detected in the last iteration?
        #self.detected = detected  
        
        #polynomial coefficients for the most recent fit
        self.current_fit = current_fit#[np.array([False])]  
        
        # x values of the last n fits of the line
        self.recent_xfitted.append(current_fit)# = [] #?
        
        #x values for detected line pixels
        self.allx = detected_x #? None  
        #y values for detected line pixels
        self.ally = detected_y #?  None

# Assuming you have created a warped binary image called "binary_warped"
def find_lines(binary_warped):

    # Choose the number of sliding windows
    nwindows = 9
    # Set the width of the windows +/- margin
    margin = 100
    #margin = 150
    # Set minimum number of pixels found to recenter window
    minpix = 50
    
    # Take a histogram of the bottom half of the image
    #histogram = np.sum(binary_warped[binary_warped.shape[0]/2:,:], axis=0)
    histogram = np.sum(binary_warped[np.int(binary_warped.shape[0]/2):,:], axis=0)

    # Create an output image to draw on and  visualize the result
    out_img = np.dstack((binary_warped, binary_warped, binary_warped))*255
    # Find the peak of the left and right halves of the histogram
    # These will be the starting point for the left and right lines
    midpoint = np.int(histogram.shape[0]/2)
    leftx_base = np.argmax(histogram[:midpoint])
    rightx_base = np.argmax(histogram[midpoint:]) + midpoint

    # Set height of windows
    window_height = np.int(binary_warped.shape[0]/nwindows)
    # Identify the x and y positions of all nonzero pixels in the image
    nonzero = binary_warped.nonzero()
    nonzeroy = np.array(nonzero[0])
    nonzerox = np.array(nonzero[1])
    # Current positions to be updated for each window
    leftx_current = leftx_base
    rightx_current = rightx_base

    # Create empty lists to receive left and right lane pixel indices
    left_lane_inds = []
    right_lane_inds = []

    # Step through the windows one by one
    right_stop = False
    left_stop = False
    for window in range(nwindows):
        # Identify window boundaries in x and y (and right and left)
        win_y_low = binary_warped.shape[0] - (window+1)*window_height
        win_y_high = binary_warped.shape[0] - window*window_height
        win_xleft_low = leftx_current - margin
        win_xleft_high = leftx_current + margin
        win_xright_low = rightx_current - margin
        win_xright_high = rightx_current + margin
        # Draw the windows on the visualization image
        cv2.rectangle(out_img,(win_xleft_low,win_y_low),(win_xleft_high,win_y_high),
        (0,255,0), 2) 
        cv2.rectangle(out_img,(win_xright_low,win_y_low),(win_xright_high,win_y_high),
        #(0,255,0), 2) 
        (0,0,255), 2) 
        # Identify the nonzero pixels in x and y within the window
        good_left_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & 
        (nonzerox >= win_xleft_low) &  (nonzerox < win_xleft_high)).nonzero()[0]
        
        good_right_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & 
        (nonzerox >= win_xright_low) &  (nonzerox < win_xright_high)).nonzero()[0]
        # Append these indices to the lists
        ##if left_stop == False:
        left_lane_inds.append(good_left_inds)
        ##if right_stop == False:
        right_lane_inds.append(good_right_inds)
        # If you found > minpix pixels, recenter next window on their mean position
        if len(good_left_inds) > minpix:
            leftx_current = np.int(np.mean(nonzerox[good_left_inds]))
        if len(good_right_inds) > minpix:        
            rightx_current = np.int(np.mean(nonzerox[good_right_inds]))
        
        # If 
        ##if (len(nonzerox[good_left_inds]) < 1 and window > nwindows//2) :
        ##    left_stop = True
        ##if (len(nonzerox[good_right_inds]) < 1 and window > nwindows//2) :
        ##    right_stop = True
        #print(len(nonzerox[good_left_inds]), nwindows//2, window,( window > nwindows//2) )

    # Concatenate the arrays of indices
    left_lane_inds = np.concatenate(left_lane_inds)
    right_lane_inds = np.concatenate(right_lane_inds)

    # Extract left and right line pixel positions
    leftx = nonzerox[left_lane_inds]
    lefty = nonzeroy[left_lane_inds] 
    rightx = nonzerox[right_lane_inds]
    righty = nonzeroy[right_lane_inds] 

    # Fit a second order polynomial to each
    left_fit = np.polyfit(lefty, leftx, 2)
    right_fit = np.polyfit(righty, rightx, 2)

    right_line = Line()
    left_line = Line()
    right_line.update_average_and_best()
    right_line.add_recent_data(right_fit, rightx, righty)
    left_line.add_recent_data(left_fit, leftx, lefty)
    
    # was the line detected in the last iteration?
    right_line.detected = False  
    # x values of the last n fits of the line
    right_line.recent_xfitted = [] 
    #average x values of the fitted line over the last n iterations
    right_line.bestx = None     
    #polynomial coefficients averaged over the last n iterations
    right_line.best_fit = None  
    #polynomial coefficients for the most recent fit
    right_line.current_fit = right_fit #[np.array([False])]  
    #radius of curvature of the line in some units
    right_line.radius_of_curvature = None 
    #distance in meters of vehicle center from the line
    right_line.line_base_pos = None 
    #difference in fit coefficients between last and new fits
    right_line.diffs = np.array([0,0,0], dtype='float') 

    
    return left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img

Skip the sliding windows step once you know where the lines are

Now I know where the lines are you have a fit! In the next frame of video I don't need to do a blind search again, but instead I can just search in a margin around the previous line position.

# Assume you now have a new warped binary image 
# from the next frame of video (also called "binary_warped")
# It's now much easier to find line pixels!
def find_lines_again(binary_warped, left_fit, right_fit):
    nonzero = binary_warped.nonzero()
    nonzeroy = np.array(nonzero[0])
    nonzerox = np.array(nonzero[1])
    margin = 100
    
    margin = 50
    #margin = 30
    
    left_lane_inds = ((nonzerox > (left_fit[0]*(nonzeroy**2) + left_fit[1]*nonzeroy + 
    left_fit[2] - margin)) & (nonzerox < (left_fit[0]*(nonzeroy**2) + 
    left_fit[1]*nonzeroy + left_fit[2] + margin))) 

    right_lane_inds = ((nonzerox > (right_fit[0]*(nonzeroy**2) + right_fit[1]*nonzeroy + 
    right_fit[2] - margin)) & (nonzerox < (right_fit[0]*(nonzeroy**2) + 
    right_fit[1]*nonzeroy + right_fit[2] + margin)))  

    # Again, extract left and right line pixel positions
    leftx = nonzerox[left_lane_inds]
    lefty = nonzeroy[left_lane_inds] 
    rightx = nonzerox[right_lane_inds]
    righty = nonzeroy[right_lane_inds]
    # Fit a second order polynomial to each
    left_fit = np.polyfit(lefty, leftx, 2)
    right_fit = np.polyfit(righty, rightx, 2)
    # Generate x and y values for plotting
    ploty = np.linspace(0, binary_warped.shape[0]-1, binary_warped.shape[0] )
    left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
    right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
    
    return left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img

# Calculate the new radii of curvature
def calc_radii_of_curvature(fit_cr, y_eval, ym_per_pix):
    curverad = ((1 + (2*fit_cr[0]*y_eval*ym_per_pix + fit_cr[1])**2)**1.5) / np.absolute(2*fit_cr[0])
    return curverad

#left_curverad = ((1 + (2*left_fit_cr[0]*y_eval*ym_per_pix + left_fit_cr[1])**2)**1.5) / np.absolute(2*left_fit_cr[0])
#right_curverad = ((1 + (2*right_fit_cr[0]*y_eval*ym_per_pix + right_fit_cr[1])**2)**1.5) / np.absolute(2*right_fit_cr[0])

# Define y-value where we want radius of curvature
# I'll choose the maximum y-value, corresponding to the bottom of the image
def calc_curverad(ploty, fit):
    y_eval = np.max(ploty)
    curverad = ((1 + (2*fit[0]*y_eval + fit[1])**2)**1.5) / np.absolute(2*fit[0])
    return curverad
# Example values: 1926.74 1908.48

def calc_radius_in_meter(y_eval, left_fit_cr, right_fit_cr):
    # Define conversions in x and y from pixels space to meters
    ym_per_pix = 30/720 # meters per pixel in y dimension
    xm_per_pix = 3.7/700 # meters per pixel in x dimension

    # Calculate the new radii of curvature
    left_curverad = ((1 + (2*left_fit_cr[0]*y_eval*ym_per_pix + left_fit_cr[1])**2)**1.5) / np.absolute(2*left_fit_cr[0])
    right_curverad = ((1 + (2*right_fit_cr[0]*y_eval*ym_per_pix + right_fit_cr[1])**2)**1.5) / np.absolute(2*right_fit_cr[0])
    # Now our radius of curvature is in meters
    return left_curverad, right_curverad
    # Example values: 632.1 m    626.2 m

def calc_vehicle_position(left_fit, right_fit, image):
    #y_btm = 720#
    y_btm = image.shape[0] # Y
    #print("y_btm", y_btm)
    #image_mid = 720//2
    image_mid = image.shape[1]/2 # X
    #print("image_mid", image_mid)
    
    x_left = left_fit[0]*y_btm**2 + left_fit[1]*y_btm + left_fit[2]
    x_right = right_fit[0]*y_btm**2 + right_fit[1]*y_btm + right_fit[2]
    
    #print("x_left", x_left)
    #print("x_right", x_right)
    
    lane_width = x_right - x_left

    x_mid = x_left + (lane_width)//2
    
    #print("x_mid", x_mid)
    
    vehicle_position = image_mid - x_mid
    
    LANE_WIDTH_METER = 3.4
    meter_per_pixel = LANE_WIDTH_METER/lane_width
    vehicle_position_meter = vehicle_position*meter_per_pixel
    
    return vehicle_position_meter#vehicle_position

def put_text(image, text, margin_from_edge = 5, font = cv2.FONT_HERSHEY_PLAIN, font_size = 0.6, color = (255,255,0)):

    #w = image.shape[0]
    #h = image.shape[1]
    
    start_x = margin_from_edge
    start_y = margin_from_edge
    
    cv2.putText(image,text,(start_x,start_y),font, font_size,color)

Visualization

def draw_rectangle(img):
    #result = cv.rectangle(img, (10,   50), (50,  150), (255, 0, 0), 3, 4)
    #image = cv2.polylines(img, np.float32(SRC_RECTANGLE), True, (0, 0, 255), 5)
    pts = np.array(SRC_TRAPEZOID_ORG_ORDR, np.int32)
    pts = pts.reshape((-1, 1, 2))
    #pts = np.float32(SRC_RECTANGLE)
    copy = img.copy()
    cv2.polylines(copy, [pts],True,(255,0,0), thickness=2)
    return copy
#X_CENTER = 410
#Y_CENTER = 610
#X_BOTTOM = 660
#X_OFFSET = 5
#Y_OFFSET = 400

class LineProjector():
    def __init__(self, Minv):
        self.Minv = Minv
        
    def project_lines(self, original_image, warped, left_fit, right_fit):
        
        ploty = np.linspace(0, warped.shape[0]-1, warped.shape[0] )
        
        left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
        right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]

        # Create an image to draw the lines on
        warp_zero = np.zeros_like(warped).astype(np.uint8)
        color_warp = np.dstack((warp_zero, warp_zero, warp_zero))

        # Recast the x and y points into usable format for cv2.fillPoly()
        pts_left = np.array([np.transpose(np.vstack([left_fitx, ploty]))])
        pts_right = np.array([np.flipud(np.transpose(np.vstack([right_fitx, ploty])))])
        pts = np.hstack((pts_left, pts_right))

        # Draw the lane onto the warped blank image
        cv2.fillPoly(color_warp, np.int_([pts]), (0,255, 0))

        # Warp the blank back to original image space using inverse perspective matrix (Minv)
        #newwarp = cv2.warpPerspective(color_warp, self.Minv, (image.shape[1], image.shape[0])) 
        newwarp = cv2.warpPerspective(color_warp, self.Minv, (original_image.shape[1], original_image.shape[0])) 
        # Combine the result with the original image
        result = cv2.addWeighted(original_image, 1, newwarp, 0.3, 0)
        return result

# Generate x and y values for plotting
def visualize_lines(warped, left_fit, right_fit, left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img):
    
    ploty = np.linspace(0, warped.shape[0]-1, warped.shape[0] )
    
    out_img[nonzeroy[left_lane_inds], nonzerox[left_lane_inds]] = [255, 0, 0]
    out_img[nonzeroy[right_lane_inds], nonzerox[right_lane_inds]] = [0, 0, 255]
    
    plt.imshow(out_img)
    
    # Generate x and y values for plotting
    left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
    right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
    plt.plot(left_fitx, ploty, color='yellow')
    plt.plot(right_fitx, ploty, color='yellow')
    
    #print(warped.shape[0], warped.shape[1])

    plt.xlim(0, 1280)
    plt.ylim(720, 0)

test_img = mpimg.imread('./test_images/test2.jpg')
image_threshed = pipeline_thresh(test_img)

binary_warped, M, Minv = pipeline_warp(image_threshed)

left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img = find_lines(binary_warped)

visualize_lines(binary_warped, left_fit, right_fit, left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img)

#print("left:", calc_radii_of_curvature(left_fit, y_eval, ym_per_pix))
#print("riht:",calc_radii_of_curvature(right_fit, y_eval, ym_per_pix))

test_img = mpimg.imread('./test_images/test2.jpg')
image_threshed = pipeline_thresh(test_img)

binary_warped, M, Minv = pipeline_warp(image_threshed)

left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img = find_lines_again(binary_warped, left_fit, right_fit)

visualize_lines(binary_warped, left_fit, right_fit, left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img)

Sliding Window Search

TODO: Convolution is not used in this solution.

def window_mask(width, height, img_ref, center,level):
    output = np.zeros_like(img_ref)
    output[int(img_ref.shape[0]-(level+1)*height):int(img_ref.shape[0]-level*height),max(0,int(center-width/2)):min(int(center+width/2),img_ref.shape[1])] = 1
    return output

def find_window_centroids(image, window_width, window_height, margin):
    
    window_centroids = [] # Store the (left,right) window centroid positions per level
    window = np.ones(window_width) # Create our window template that we will use for convolutions
    
    # First find the two starting positions for the left and right lane by using np.sum to get the vertical image slice
    # and then np.convolve the vertical image slice with the window template 
    
    # Sum quarter bottom of image to get slice, could use a different ratio
    l_sum = np.sum(image[int(3*image.shape[0]/4):,:int(image.shape[1]/2)], axis=0)
    l_center = np.argmax(np.convolve(window,l_sum))-window_width/2
    r_sum = np.sum(image[int(3*image.shape[0]/4):,int(image.shape[1]/2):], axis=0)
    r_center = np.argmax(np.convolve(window,r_sum))-window_width/2+int(image.shape[1]/2)
    
    # Add what we found for the first layer
    window_centroids.append((l_center,r_center))
    
    # Go through each layer looking for max pixel locations
    for level in range(1,(int)(image.shape[0]/window_height)):
	    # convolve the window into the vertical slice of the image
	    image_layer = np.sum(image[int(image.shape[0]-(level+1)*window_height):int(image.shape[0]-level*window_height),:], axis=0)
	    conv_signal = np.convolve(window, image_layer)
	    # Find the best left centroid by using past left center as a reference
	    # Use window_width/2 as offset because convolution signal reference is at right side of window, not center of window
	    offset = window_width/2
	    l_min_index = int(max(l_center+offset-margin,0))
	    l_max_index = int(min(l_center+offset+margin,image.shape[1]))
	    l_center = np.argmax(conv_signal[l_min_index:l_max_index])+l_min_index-offset
	    # Find the best right centroid by using past right center as a reference
	    r_min_index = int(max(r_center+offset-margin,0))
	    r_max_index = int(min(r_center+offset+margin,image.shape[1]))
	    r_center = np.argmax(conv_signal[r_min_index:r_max_index])+r_min_index-offset
	    # Add what we found for that layer
	    window_centroids.append((l_center,r_center))

    return window_centroids

def gen_output(window_centroids, warped):
    # If we found any window centers
    output = None
    if len(window_centroids) > 0:

        # Points used to draw all the left and right windows
        l_points = np.zeros_like(warped)
        r_points = np.zeros_like(warped)

        # Go through each level and draw the windows 	
        for level in range(0,len(window_centroids)):
            # Window_mask is a function to draw window areas
            l_mask = window_mask(window_width,window_height,warped,window_centroids[level][0],level)
            r_mask = window_mask(window_width,window_height,warped,window_centroids[level][1],level)
            # Add graphic points from window mask here to total pixels found 
            l_points[(l_points == 255) | ((l_mask == 1) ) ] = 255
            r_points[(r_points == 255) | ((r_mask == 1) ) ] = 125#255

        # Draw the results
        template = np.array(r_points+l_points,np.uint8) # add both left and right window pixels together
        zero_channel = np.zeros_like(template) # create a zero color channel
        template = np.array(cv2.merge((zero_channel,template,zero_channel)),np.uint8) # make window pixels green
        warpage= np.dstack((warped, warped, warped))*255 # making the original road pixels 3 color channels
        output = cv2.addWeighted(warpage, 1, template, 0.5, 0.0) # overlay the orignal road image with window results

    # If no window centers found, just display orginal road image
    else:
        output = np.array(cv2.merge((warped,warped,warped)),np.uint8)
    return output

# Read in a thresholded image
#warped = mpimg.imread('warped_example.jpg')

test_img = mpimg.imread('./test_images/test2.jpg')
image_threshed = pipeline_thresh(test_img)
binary_warped, M, Minv = pipeline_warp(image_threshed)

# window settings
window_width = 50 
window_height = 80 # Break image into 9 vertical layers since image height is 720
margin = 100 # How much to slide left and right for searching

window_centroids = find_window_centroids(binary_warped, window_width, window_height, margin)

output = gen_output(window_centroids, binary_warped)
# Display the final results
plt.imshow(output)
plt.title('window fitting results')
plt.show()

Processing Each Image

Y_TOP = 450#435#440
Y_BTM = 668

X_CTR = 640#640 = 1280/2

X_TOP_WDT = 120#48#45#50#55#65#50#50#60#60
X_BTM_WDT = 1000#1280#1000#1000#900#850

Y_TOP = 460#435#440
X_TOP_WDT = 150#48#45#50#55#65#50#50#60#60

#X_BTM_WDT = 900

TOP_LEFT = (X_CTR - X_TOP_WDT/2 , Y_TOP)
TOP_RIGHT = (X_CTR + X_TOP_WDT/2, Y_TOP)
BTM_LEFT = (X_CTR - X_BTM_WDT/2 , Y_BTM)
BTM_RIGHT = (X_CTR + X_BTM_WDT/2, Y_BTM)

SRC_TRAPEZOID_ORG_ORDR = [TOP_LEFT, TOP_RIGHT, BTM_RIGHT, BTM_LEFT]
#SRC_TRAPEZOID_WRP = [TOP_LEFT, TOP_RIGHT, BTM_LEFT, BTM_RIGHT]

def show_process_images(image, objpoints, imgpoints, left_fit_prev=None, right_fit_prev=None, sx_thresh=(20, 100)):
    
    #put_text(image, "test")
    #cv2.putText(image,text,(start_x,start_y),font, font_size,color)
    image = cv2.putText(image,"text",(10,10),fontFace = cv2.FONT_HERSHEY_PLAIN, fontScale = 10, color = (255,255,0))
    
    undistorted = cal_undistort(image, objpoints, imgpoints)
    #show_two_images(image, undistorted)
    thresed = pipeline_thresh(undistorted, sx_thresh=sx_thresh)
    #show_two_images(undistorted, thresed)
    binary_warped, M, Minv = pipeline_warp(thresed)
    undistorted_w_line = draw_rectangle(undistorted)

    #show_two_images(undistorted_w_line, binary_warped)
    if not (left_fit_prev==None and right_fit_prev==None):
        left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img = find_lines_again(binary_warped, left_fit_prev, right_fit_prev)
    else:
        left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img = find_lines(binary_warped)

    line_projector = LineProjector(Minv)

    #visualize_lines(ploty, left_fit, right_fit, left_lane_inds, right_lane_inds, nonzerox, nonzeroy)

    result = line_projector.project_lines(undistorted_w_line, binary_warped, left_fit, right_fit)
    #result = project_lines(org_image, warped, left_fit, right_fit, ploty, Minv)
    #plt.imshow(result)
    show_two_images(thresed, binary_warped, title1="Original Image", title2="Warped Image")
    empty = None
    
    ploty = np.linspace(0, image.shape[0]-1, image.shape[0] )
    y_eval = np.max(ploty)
    left_curverad, right_curverad = calc_radius_in_meter(y_eval, left_fit, right_fit)
    #print(left_curverad, right_curverad)
    
    vehicle_position = calc_vehicle_position(left_fit, right_fit, binary_warped)
    
    text_radius = "Radius:{} ".format((left_curverad + right_curverad)/2)
    text_position = "Vehicle Position:{} ".format(vehicle_position)
    #put_text(result, text)
    print(text_position)
    cv2.putText(result,text_radius,(50,100),fontFace = cv2.FONT_ITALIC, fontScale = 3, color = (0,0,255))#255,255,255))
    cv2.putText(result,text_position,(50,200),fontFace = cv2.FONT_ITALIC, fontScale = 3, color = (0,255,0))#255,255,255))
    
    show_two_images(result, result)
        
    left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img = find_lines(binary_warped)
    
    visualize_lines(binary_warped, left_fit, right_fit, left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img)

    return left_fit, right_fit

test_img = mpimg.imread('./test_images/straight_lines1.jpg')
#print(test_img.shape[0])
#print(test_img.shape[1])

show_process_images(test_img, objpoints, imgpoints)

Vehicle Position:-0.07515782717252985 

(array([  1.75148122e-05,   1.39427219e-02,   1.07712657e+02]),
 array([  4.26978875e-05,  -4.98363521e-02,   6.30557263e+02]))

test_img = mpimg.imread('./test_images/test1.jpg')
show_process_images(test_img, objpoints, imgpoints)

Vehicle Position:-0.1981039113418875 

(array([  1.57252064e-04,  -1.68836315e-01,   1.83593740e+02]),
 array([  1.12255570e-05,  -1.43115345e-01,   7.30788397e+02]))

test_img = mpimg.imread('./test_images/test2.jpg')
show_process_images(test_img, objpoints, imgpoints)

Vehicle Position:-0.34809544784343893 

(array([ -1.76637121e-04,   2.81933131e-01,   5.85529986e+01]),
 array([ -1.59462371e-04,   1.96172746e-01,   5.89768678e+02]))

test_img = mpimg.imread('./test_images/test4.jpg')
left_fit_prev, right_fit_prev = show_process_images(test_img, objpoints, imgpoints)

Vehicle Position:-0.35626406703627606 

TODO: Consider the following

The approach to better this is to

Try other color spaces. For instance looking at the blue and green channels of the RGB might give you yellow pixels easily (yellow = blue + green right) Try to limit your search based on the expected lane width. This will prevent the weird effects that you see in your final image. In every detection, reinforce the weaker lane (lane with lesser pixels) with the stronger lane by assuming that the two lanes are parallel.

If there are too few pixels detected in a particular lane, just discard it and use the previous lane as the value. One another approach I took was to do a weighted average of the polynomials from previous frames to current one, based on the number of thresholded pixels. So if the current image has a lot of thresholded pixels, then I give the current polynomial a lot of importance, otherwise no. This is a generalization of point 3.

test_img = mpimg.imread('./issue_images/org_video_image_20171217_10_22_06.jpg')
#test_output = pipeline_thresh(test_img, s_thresh=(170, 255), sx_thresh=(5, 100))
show_process_images(test_img, objpoints, imgpoints)#, sx_thresh=(5, 100))
show_process_images(test_img, objpoints, imgpoints,left_fit_prev, right_fit_prev)

Vehicle Position:0.7483220157717451 

C:\Anaconda3\envs\carnd-term1\lib\site-packages\ipykernel_launcher.py:15: FutureWarning: comparison to `None` will result in an elementwise object comparison in the future.
  from ipykernel import kernelapp as app

Vehicle Position:-2.190518855984685 

(array([  7.00593440e-04,  -6.70527130e-01,   3.68619736e+02]),
 array([  3.38328657e-04,  -5.03670614e-01,   5.90633562e+02]))

Thresholded binary image

• Apply a distortion correction to raw images.
• Use color transforms, gradients, etc., to create a thresholded binary image.

Perspective transform to rectify binary image ("birds-eye view")

• Apply a perspective transform to rectify binary image ("birds-eye view").
• Detect lane pixels and fit to find the lane boundary.

Lane boundaries and numerical estimation of lane curvature and vehicle position

• Determine the curvature of the lane and vehicle position with respect to center.
• Warp the detected lane boundaries back onto the original image.
• Output visual display of the lane boundaries and numerical estimation of lane curvature and vehicle position.

flg_in_process = False
left_fit = None
right_fit = None
count = 0
DIR_ORG_VIDEO_SHOT = "./video_images/"
DIR_CVT_VIDEO_SHOT = "./converted_images/"
ORG_VIDEO_FILE_NAME = "org_video_image_"
CVT_VIDEO_FILE_NAME = "cvt_video_image_"
from PIL import Image
def save_image(image, dirname, filename):
    """save a image file"""    
    filepath = dirname + filename + datetime.now().strftime("%Y%m%d_%H_%M_%S.jpg")
    if not os.path.exists(filepath) :
        Image.fromarray(image).save(filepath)

from datetime import datetime
import os
def video_pipeline(image):

    #save_image(image,DIR_ORG_VIDEO_SHOT, ORG_VIDEO_FILE_NAME)
    
    processed_image = np.copy(image)
    
    # Undistort image (Try)
    processed_image = cal_undistort(processed_image, objpoints, imgpoints)
    
    processed_image = pipeline_thresh(processed_image)
    binary_warped, M, Minv = pipeline_warp(processed_image)
    
    
    if flg_in_process == False:
        left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img = find_lines(binary_warped)
    else:
        left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img = find_lines_again(binary_warped, left_fit, right_fit)
    #ploty = np.linspace(0, binary_warped.shape[0]-1, binary_warped.shape[0] )
    line_projector = LineProjector(Minv)
    processed_image = line_projector.project_lines(image, binary_warped, left_fit, right_fit)#, ploty, Minv)
    
    ploty = np.linspace(0, image.shape[0]-1, image.shape[0] )
    y_eval = np.max(ploty)
    left_curverad, right_curverad = calc_radius_in_meter(y_eval, left_fit, right_fit)    
    #text = "Radius:{} ".format((left_curverad + right_curverad)/2)
    #cv2.putText(processed_image,text,(50,200),fontFace = cv2.FONT_ITALIC, fontScale = 3, color = (0,0,255))#255,255,255))
    
    vehicle_position = calc_vehicle_position(left_fit, right_fit, binary_warped)
    
    text_radius = "Radius:{} ".format((left_curverad + right_curverad)/2)
    text_position = "Vehicle Position:{} ".format(vehicle_position)
    cv2.putText(processed_image,text_radius,(50,100),fontFace = cv2.FONT_ITALIC, fontScale = 3, color = (0,0,255))#255,255,255))
    cv2.putText(processed_image,text_position,(50,200),fontFace = cv2.FONT_ITALIC, fontScale = 3, color = (0,255,0))#255,255,255))
    
    
    #save_image(processed_image,DIR_CVT_VIDEO_SHOT, CVT_VIDEO_FILE_NAME)
    
    return processed_image # This must be a color image

def process_image(image):
    # NOTE: The output you return should be a color image (3 channel) for processing video below
    # TODO: put your pipeline here,
    # you should return the final output (image where lines are drawn on lanes)
    result = video_pipeline(image)
    return result

# Import everything needed to edit/save/watch video clips
from moviepy.editor import VideoFileClip
from IPython.display import HTML

VIDEO_INPUT = 'challenge_video.mp4'
VIDEO_OUTPUT = 'output_images/challenge_video_output.mp4'
## To speed up the testing process you may want to try your pipeline on a shorter subclip of the video
## To do so add .subclip(start_second,end_second) to the end of the line below
## Where start_second and end_second are integer values representing the start and end of the subclip
## You may also uncomment the following line for a subclip of the first 5 seconds
##clip1 = VideoFileClip("test_videos/solidWhiteRight.mp4").subclip(0,5)
clip1 = VideoFileClip(VIDEO_INPUT)
white_clip = clip1.fl_image(process_image) #NOTE: this function expects color images!!
%time white_clip.write_videofile(VIDEO_OUTPUT, audio=False)
flg_in_process = False
left_fit = None
right_fit = None

[MoviePy] >>>> Building video output_images/challenge_video_output.mp4
[MoviePy] Writing video output_images/challenge_video_output.mp4

100%|████████████████████████████████████████████████████████████████████████████████| 485/485 [06:32<00:00,  1.32it/s]

[MoviePy] Done.
[MoviePy] >>>> Video ready: output_images/challenge_video_output.mp4 

Wall time: 6min 34s

HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format(VIDEO_OUTPUT))